package dblp.crawler;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.CDATASection;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import repositories.AuthorRepository;
import repositories.PaperRepository;
import repositories.SeedRepository;
import domain.Author;
import domain.AuthorNamePair;
import domain.Paper;
import domain.Seed;
import domain.SeedStatus;

public class DBLPCrawler extends Thread {
	public final String SEARCH_URL = "http://www.dblp.org/search/api/?q=%s*&h=1000&c=4&f=0&format=xml";

	private static Lock seedLock = new ReentrantLock();
	private static Lock paperLock = new ReentrantLock();

	private static Map<String, Map<String, UUID>> authorIds = Collections.synchronizedMap(new HashMap<String, Map<String, UUID>>());

	private AuthorRepository authorRepository = new AuthorRepository();
	private SeedRepository seedRepository = new SeedRepository();
	private PaperRepository paperRepository = new PaperRepository();

	private Seed seed = null;

	public DBLPCrawler(ThreadGroup threadGroup, String name, Seed seed) throws Exception {
		super(threadGroup, name);

		if (seed == null)
			throw new Exception("Seed cannot be null.");

		this.seed = seed;
	}

	@Override
	public void run() {
		try {
			outputMessage(String.format("Started Crawling '%s'", seed.getSeed().toUpperCase()));

			Document doc = getXML(seed.getSeed());
			parseXML(doc);

			seed.setStatus(SeedStatus.Crawled);
			seedRepository.saveSeed(seed);

			outputMessage(String.format("Finished Crawling '%s'", seed.getSeed().toUpperCase()));
		} catch (Exception ex) {
			ex.printStackTrace();

			try {
				if (seed != null && seed.getStatus() != SeedStatus.Crawled) {
					seed.setStatus(SeedStatus.CrawlingError);
					seed.setError(ex.getMessage());
					seedRepository.saveSeed(seed);
				}
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}

	}

	private void outputMessage(String message) {
		String prepend = String.format("[%s | %s]", this.getName(), seed.getSeed().toUpperCase());

		System.out.println(StringUtils.rightPad(prepend, 40) +  " " + message);
	}


	private Document getXML(String name) {
		Document result = null;
		String requestName = String.format("ce:author:%s:", name.toLowerCase()
				.replaceAll("\\s+", "_"));

		String requestUrl = String.format(SEARCH_URL, requestName);
		URL resourceUrl;
		try {
			resourceUrl = new URL(requestUrl);

			// UTF-8 Fix - http://www.mkyong.com/java/sax-error-malformedbytesequenceexception-invalid-byte-1-of-1-byte-utf-8-sequence/
			Reader reader = new InputStreamReader(resourceUrl.openStream(), "UTF-8");
			InputSource source = new InputSource(reader);
			source.setEncoding("UTF-8");			

			DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
			DocumentBuilder db = dbf.newDocumentBuilder();

			result = db.parse(source);

		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (SAXException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return result;
	}

	private void parseXML(Document doc) throws Exception {
		Element root = doc.getDocumentElement();

		NodeList hits = root.getElementsByTagName("hit");
		if (hits != null && hits.getLength() > 0) {
			for (int i = 0; i < hits.getLength(); i++) {
				Element hit = (Element) hits.item(i);

				Element dataElement = (Element) (hit
						.getElementsByTagName("title").item(0));
				Element urlElement = (Element) (hit.getElementsByTagName("url")
						.item(0));

				String data = ((CDATASection) dataElement.getFirstChild())
						.getData();
				data = String
						.format("<?xml version=\"1.0\" encoding=\"UTF-8\"?><dblp>%s</dblp>",
								data);
				Document dataDocument = loadXMLFromString(data);
				Element dataRoot = dataDocument.getDocumentElement();

				NodeList authorsList = dataRoot
						.getElementsByTagName("dblp:author");
				Element titleElement = getElement(dataRoot, "dblp:title");
				//Element venueElement = getElement(dataRoot, "dblp:venue");
				Element yearElement = getElement(dataRoot, "dblp:year");
				//Element typeElement = getElement(dataRoot, "dblp:type");

				// Create Paper
				Paper paper = new Paper();

				paper.setTitle(titleElement.getTextContent());
				paper.setDblpUrl(urlElement.getTextContent());
				paper.setDocumentUrl(titleElement.getAttribute("ee"));
				paper.setYear(Integer.parseInt(yearElement.getTextContent()));

				Vector<String> authorNames = new Vector<String>();

				for (int j = 0; j < authorsList.getLength(); j++) {
					authorNames.add(
							cleanAuthorName(
									((Element) authorsList.item(j)).getTextContent()));
				}

				List<Author> authors = getAuthors(authorNames);

				for(Author author : authors) {
					paper.addAuthor(author);

					if (this.seedRepository.createSeed(author.getFullName(), seed)) {
						outputMessage(String.format("    New Author Seed '%s'", author.getFullName()));
					}
				}

				Thread.yield();

				outputMessage(String.format("    Adding Paper '%s'", paper.getTitle()));
				paperLock.lock();
				this.paperRepository.createPaper(paper);
				paperLock.unlock();

				/*
				 * System.out.println("Title: " + ); System.out.println("ee: " +
				 * titleElement.getAttribute("ee")); System.out.println("url: "
				 * + venueElement.getAttribute("url"));
				 * System.out.println("Venue: " +
				 * venueElement.getTextContent()); System.out.println("Year: " +
				 * ); System.out.println("Type: " +
				 * typeElement.getTextContent());
				 */
			}
		}
	}

	private List<Author> getAuthors(List<String> authorNames) throws Exception {
		Vector<Author> result = new Vector<Author>();

		for(String authorName : authorNames) {
			Author author = null;
			Map<String, UUID> coauthorMap = null;

			if (DBLPCrawler.authorIds.containsKey(authorName)) {
				UUID authorId = null;
				coauthorMap = DBLPCrawler.authorIds.get(authorName);

				for(String coauthorName : authorNames) {
					if (!coauthorName.equalsIgnoreCase(authorName) && coauthorMap.containsKey(coauthorName)) {
						authorId = coauthorMap.get(coauthorName);
						break;
					}
				}

				if (authorId != null)
					author = authorRepository.getAuthor(authorId);
			}

			if (author == null) {
				author = new Author(authorName);
				authorRepository.saveAuthor(author);
			}

			if (coauthorMap == null)
				coauthorMap = Collections.synchronizedMap(new HashMap<String, UUID>());

			for(String coauthorName : authorNames) {
				if (!authorName.equalsIgnoreCase(coauthorName) && !coauthorMap.containsKey(coauthorName)) {
					coauthorMap.put(coauthorName, author.getAuthorID());
				}
			}

			if (!DBLPCrawler.authorIds.containsKey(authorName))
				DBLPCrawler.authorIds.put(authorName, coauthorMap);

			result.add(author);
		}

		return result;
	}

	private String cleanAuthorName(String authorName) {
		return authorName.trim().replaceAll("\\s+", " ").toUpperCase();
	}

	private Element getElement(Element parent, String element) {
		NodeList nodes = parent.getElementsByTagName(element);

		if (nodes != null && nodes.getLength() > 0)
			return (Element) nodes.item(0);

		return null;
	}

	private Document loadXMLFromString(String xml) {
		xml = fixXml(xml);

		Document result = null;
		try {
			DocumentBuilderFactory factory = DocumentBuilderFactory
					.newInstance();
			DocumentBuilder builder;

			builder = factory.newDocumentBuilder();
			InputSource is = new InputSource(new StringReader(xml));

			result = builder.parse(is);
		} catch (ParserConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (SAXException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.out.println(xml);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return result;
	}

	private String fixXml(String xml) {
		StringBuffer output = new StringBuffer();
		Pattern attributeRegex = Pattern.compile("=\"(.*?)\"");
		Pattern valueRegex = Pattern.compile(">([^<]+?)<");

		// Do Attributes
		Matcher matcher = attributeRegex.matcher(xml);
		while (matcher.find()) {
			String replacement = matcher.group(0);
			replacement = StringUtils.removeStart(replacement, "=\"");
			replacement = StringUtils.removeEnd(replacement, "\"");

			replacement = StringEscapeUtils.escapeXml(replacement);
			replacement = String.format("=\"%s\"", replacement);

			matcher.appendReplacement(output, replacement);
		}
		matcher.appendTail(output);

		xml = output.toString();
		output = new StringBuffer();

		matcher = valueRegex.matcher(xml);
		while (matcher.find()) {
			String replacement = matcher.group(0);
			replacement = StringUtils.removeStart(replacement, ">");
			replacement = StringUtils.removeEnd(replacement, "<");

			replacement = StringEscapeUtils.escapeXml(replacement);
			replacement = String.format(">%s<", replacement);

			matcher.appendReplacement(output, replacement);
		}
		matcher.appendTail(output);

		return output.toString();
	}
}